From 169a90fe31399391f9944e6c6f6422d6409613f5 Mon Sep 17 00:00:00 2001 From: Jeroen van der Heijden Date: Wed, 16 Sep 2020 15:37:07 +0200 Subject: [PATCH] added timeval and interval functions, #138 --- grammar/grammar.py | 12 +++- include/siri/grammar/grammar.h | 6 +- include/siri/version.h | 2 +- itest/test_select.py | 47 +++++++++++++++ src/siri/db/aggregate.c | 107 +++++++++++++++++++++++++++++++++ src/siri/grammar/grammar.c | 24 +++++++- 6 files changed, 192 insertions(+), 6 deletions(-) diff --git a/grammar/grammar.py b/grammar/grammar.py index fca6316d..11eef64f 100644 --- a/grammar/grammar.py +++ b/grammar/grammar.py @@ -98,6 +98,7 @@ class SiriGrammar(Grammar): Token('&'), Keyword('intersection'), most_greedy=False) + k_interval = Keyword('interval') k_ip_support = Keyword('ip_support') k_last = Keyword('last') k_length = Keyword('length') @@ -164,9 +165,10 @@ class SiriGrammar(Grammar): k_tag = Keyword('tag') k_tags = Keyword('tags') k_tee_pipe_name = Keyword('tee_pipe_name') + k_time_precision = Keyword('time_precision') k_timeit = Keyword('timeit') + k_timeval = Keyword('timeval') k_timezone = Keyword('timezone') - k_time_precision = Keyword('time_precision') k_to = Keyword('to') k_true = Keyword('true') k_type = Keyword('type') @@ -521,6 +523,12 @@ class SiriGrammar(Grammar): f_last = Sequence( k_last, '(', Optional(time_expr), ')') + f_timeval = Sequence( + k_timeval, + '(', ')') + f_interval = Sequence( + k_interval, + '(', ')') f_filter = Sequence( k_filter, @@ -574,6 +582,8 @@ class SiriGrammar(Grammar): f_stddev, f_first, f_last, + f_timeval, + f_interval, f_difference, f_derivative, f_filter, diff --git a/include/siri/grammar/grammar.h b/include/siri/grammar/grammar.h index 362d5e74..ee629c9a 100644 --- a/include/siri/grammar/grammar.h +++ b/include/siri/grammar/grammar.h @@ -5,7 +5,7 @@ * should be used with the libcleri module. * * Source class: SiriGrammar - * Created at: 2020-06-17 15:21:09 + * Created at: 2020-09-15 15:01:47 */ #ifndef CLERI_EXPORT_SIRI_GRAMMAR_GRAMMAR_H_ #define CLERI_EXPORT_SIRI_GRAMMAR_GRAMMAR_H_ @@ -60,6 +60,7 @@ enum cleri_grammar_ids { CLERI_GID_F_DIFFERENCE, CLERI_GID_F_FILTER, CLERI_GID_F_FIRST, + CLERI_GID_F_INTERVAL, CLERI_GID_F_LAST, CLERI_GID_F_LIMIT, CLERI_GID_F_MAX, @@ -72,6 +73,7 @@ enum cleri_grammar_ids { CLERI_GID_F_PVARIANCE, CLERI_GID_F_STDDEV, CLERI_GID_F_SUM, + CLERI_GID_F_TIMEVAL, CLERI_GID_F_VARIANCE, CLERI_GID_GRANT_STMT, CLERI_GID_GRANT_USER, @@ -171,6 +173,7 @@ enum cleri_grammar_ids { CLERI_GID_K_INSERT, CLERI_GID_K_INTEGER, CLERI_GID_K_INTERSECTION, + CLERI_GID_K_INTERVAL, CLERI_GID_K_IP_SUPPORT, CLERI_GID_K_LAST, CLERI_GID_K_LENGTH, @@ -233,6 +236,7 @@ enum cleri_grammar_ids { CLERI_GID_K_TAGS, CLERI_GID_K_TEE_PIPE_NAME, CLERI_GID_K_TIMEIT, + CLERI_GID_K_TIMEVAL, CLERI_GID_K_TIMEZONE, CLERI_GID_K_TIME_PRECISION, CLERI_GID_K_TO, diff --git a/include/siri/version.h b/include/siri/version.h index b834c833..95c227fc 100644 --- a/include/siri/version.h +++ b/include/siri/version.h @@ -15,7 +15,7 @@ * Note that debian alpha packages should use versions like this: * 2.0.34-0alpha0 */ -#define SIRIDB_VERSION_PRE_RELEASE "-alpha-1" +#define SIRIDB_VERSION_PRE_RELEASE "-alpha-2" #ifndef NDEBUG #define SIRIDB_VERSION_BUILD_RELEASE "+debug" diff --git a/itest/test_select.py b/itest/test_select.py index 6e3879cf..130f464f 100644 --- a/itest/test_select.py +++ b/itest/test_select.py @@ -363,6 +363,53 @@ class TestSelect(TestBase): '2-log': [[1471254712, 'log line two']] }) + self.assertEqual( + await self.client0.query('select timeval() from "aggr"'), + {'aggr': [ + [1447249033, 1447249033], + [1447249337, 1447249337], + [1447249633, 1447249633], + [1447249937, 1447249937], + [1447250249, 1447250249], + [1447250549, 1447250549], + [1447250868, 1447250868], + [1447251168, 1447251168], + [1447251449, 1447251449], + [1447251749, 1447251749], + [1447252049, 1447252049], + [1447252349, 1447252349], + [1447252649, 1447252649], + [1447252968, 1447252968], + [1447253244, 1447253244], + [1447253549, 1447253549], + [1447253849, 1447253849], + [1447254149, 1447254149], + [1447254449, 1447254449], + [1447254748, 1447254748]]}) + + self.assertEqual( + await self.client0.query('select interval() from "aggr"'), + {'aggr': [ + [1447249337, 304], + [1447249633, 296], + [1447249937, 304], + [1447250249, 312], + [1447250549, 300], + [1447250868, 319], + [1447251168, 300], + [1447251449, 281], + [1447251749, 300], + [1447252049, 300], + [1447252349, 300], + [1447252649, 300], + [1447252968, 319], + [1447253244, 276], + [1447253549, 305], + [1447253849, 300], + [1447254149, 300], + [1447254449, 300], + [1447254748, 299]]}) + self.assertEqual( await self.client0.query('select difference() from "one"'), {'one': []}) diff --git a/src/siri/db/aggregate.c b/src/siri/db/aggregate.c index dd4c5bb0..8ff99dbf 100644 --- a/src/siri/db/aggregate.c +++ b/src/siri/db/aggregate.c @@ -61,6 +61,12 @@ static siridb_points_t * AGGREGATE_derivative( static siridb_points_t * AGGREGATE_difference( siridb_points_t * source, char * err_msg); +static siridb_points_t * AGGREGATE_timeval( + siridb_points_t * source, + char * err_msg); +static siridb_points_t * AGGREGATE_interval( + siridb_points_t * source, + char * err_msg); static siridb_points_t * AGGREGATE_filter( siridb_points_t * source, siridb_aggr_t * aggr, @@ -290,6 +296,14 @@ vec_t * siridb_aggregate_list(cleri_children_t * children, char * err_msg) aggr->gid = CLERI_GID_F_LAST; break; + case CLERI_GID_K_TIMEVAL: + aggr->gid = CLERI_GID_F_TIMEVAL; + break; + + case CLERI_GID_K_INTERVAL: + aggr->gid = CLERI_GID_F_INTERVAL; + break; + default: assert (0); break; @@ -300,6 +314,17 @@ vec_t * siridb_aggregate_list(cleri_children_t * children, char * err_msg) break; + case CLERI_GID_F_TIMEVAL: + case CLERI_GID_F_INTERVAL: + AGGR_NEW + { + aggr->timespan = 1; + aggr->group_by = 0; + } + + VEC_APPEND + + break; case CLERI_GID_F_FILTER: AGGR_NEW { @@ -508,6 +533,12 @@ siridb_points_t * siridb_aggregate_run( case CLERI_GID_F_FILTER: return AGGREGATE_filter(source, aggr, err_msg); + case CLERI_GID_F_INTERVAL: + return AGGREGATE_interval(source, err_msg); + + case CLERI_GID_F_TIMEVAL: + return AGGREGATE_timeval(source, err_msg); + default: return AGGREGATE_to_one(source, aggr, err_msg); } @@ -797,6 +828,80 @@ static siridb_points_t * AGGREGATE_difference( return points; } +static siridb_points_t * AGGREGATE_interval( + siridb_points_t * source, + char * err_msg) +{ + size_t len = source->len - 1; + siridb_points_t * points = siridb_points_new(len, TP_INT); + + if (points == NULL) + { + sprintf(err_msg, "Memory allocation error."); + } + else + { + points->len = len; + + if (len) + { + siridb_point_t * prev = source->data; + siridb_point_t * curr = prev + 1; + siridb_point_t * end = source->data + source->len; + siridb_point_t * dest = points->data; + + for (; curr < end; ++prev, ++curr, ++dest) + { + uint64_t diff = curr->ts - prev->ts; + if (diff > LLONG_MAX) + { + sprintf(err_msg, + "Overflow detected while using difference()."); + siridb_points_free(points); + return NULL; + } + dest->ts = curr->ts; + dest->val.int64 = (int64_t) diff; + } + } + } + return points; +} + +static siridb_points_t * AGGREGATE_timeval( + siridb_points_t * source, + char * err_msg) +{ + siridb_points_t * points = siridb_points_new(source->len, TP_INT); + + if (points == NULL) + { + sprintf(err_msg, "Memory allocation error."); + } + else + { + siridb_point_t * curr = source->data; + siridb_point_t * end = source->data + source->len; + siridb_point_t * dest = points->data; + + points->len = source->len; + + for (; curr < end; ++curr, ++dest) + { + if (curr->ts > LLONG_MAX) + { + sprintf(err_msg, + "Overflow detected while using difference()."); + siridb_points_free(points); + return NULL; + } + dest->ts = curr->ts; + dest->val.int64 = (int64_t) curr->ts; + } + } + return points; +} + static int AGGREGATE_regex_cmp(siridb_aggr_t * aggr, char * val) { int ret; @@ -1040,6 +1145,8 @@ static siridb_points_t * AGGREGATE_group_by( points = siridb_points_new(max_sz, TP_DOUBLE); break; case CLERI_GID_F_COUNT: + case CLERI_GID_F_TIMEVAL: + case CLERI_GID_F_INTERVAL: points = siridb_points_new(max_sz, TP_INT); break; case CLERI_GID_F_MEDIAN_HIGH: diff --git a/src/siri/grammar/grammar.c b/src/siri/grammar/grammar.c index 34bff80f..91b5ead7 100644 --- a/src/siri/grammar/grammar.c +++ b/src/siri/grammar/grammar.c @@ -5,7 +5,7 @@ * should be used with the libcleri module. * * Source class: SiriGrammar - * Created at: 2020-06-17 15:21:09 + * Created at: 2020-09-15 15:01:47 */ #include "siri/grammar/grammar.h" @@ -91,6 +91,7 @@ cleri_grammar_t * compile_siri_grammar_grammar(void) cleri_token(CLERI_NONE, "&"), cleri_keyword(CLERI_NONE, "intersection", CLERI_CASE_SENSITIVE) ); + cleri_t * k_interval = cleri_keyword(CLERI_GID_K_INTERVAL, "interval", CLERI_CASE_SENSITIVE); cleri_t * k_ip_support = cleri_keyword(CLERI_GID_K_IP_SUPPORT, "ip_support", CLERI_CASE_SENSITIVE); cleri_t * k_last = cleri_keyword(CLERI_GID_K_LAST, "last", CLERI_CASE_SENSITIVE); cleri_t * k_length = cleri_keyword(CLERI_GID_K_LENGTH, "length", CLERI_CASE_SENSITIVE); @@ -165,9 +166,10 @@ cleri_grammar_t * compile_siri_grammar_grammar(void) cleri_t * k_tag = cleri_keyword(CLERI_GID_K_TAG, "tag", CLERI_CASE_SENSITIVE); cleri_t * k_tags = cleri_keyword(CLERI_GID_K_TAGS, "tags", CLERI_CASE_SENSITIVE); cleri_t * k_tee_pipe_name = cleri_keyword(CLERI_GID_K_TEE_PIPE_NAME, "tee_pipe_name", CLERI_CASE_SENSITIVE); + cleri_t * k_time_precision = cleri_keyword(CLERI_GID_K_TIME_PRECISION, "time_precision", CLERI_CASE_SENSITIVE); cleri_t * k_timeit = cleri_keyword(CLERI_GID_K_TIMEIT, "timeit", CLERI_CASE_SENSITIVE); + cleri_t * k_timeval = cleri_keyword(CLERI_GID_K_TIMEVAL, "timeval", CLERI_CASE_SENSITIVE); cleri_t * k_timezone = cleri_keyword(CLERI_GID_K_TIMEZONE, "timezone", CLERI_CASE_SENSITIVE); - cleri_t * k_time_precision = cleri_keyword(CLERI_GID_K_TIME_PRECISION, "time_precision", CLERI_CASE_SENSITIVE); cleri_t * k_to = cleri_keyword(CLERI_GID_K_TO, "to", CLERI_CASE_SENSITIVE); cleri_t * k_true = cleri_keyword(CLERI_GID_K_TRUE, "true", CLERI_CASE_SENSITIVE); cleri_t * k_type = cleri_keyword(CLERI_GID_K_TYPE, "type", CLERI_CASE_SENSITIVE); @@ -1033,6 +1035,20 @@ cleri_grammar_t * compile_siri_grammar_grammar(void) cleri_optional(CLERI_NONE, time_expr), cleri_token(CLERI_NONE, ")") ); + cleri_t * f_timeval = cleri_sequence( + CLERI_GID_F_TIMEVAL, + 3, + k_timeval, + cleri_token(CLERI_NONE, "("), + cleri_token(CLERI_NONE, ")") + ); + cleri_t * f_interval = cleri_sequence( + CLERI_GID_F_INTERVAL, + 3, + k_interval, + cleri_token(CLERI_NONE, "("), + cleri_token(CLERI_NONE, ")") + ); cleri_t * f_filter = cleri_sequence( CLERI_GID_F_FILTER, 5, @@ -1083,7 +1099,7 @@ cleri_grammar_t * compile_siri_grammar_grammar(void) cleri_t * aggregate_functions = cleri_list(CLERI_GID_AGGREGATE_FUNCTIONS, cleri_choice( CLERI_NONE, CLERI_FIRST_MATCH, - 19, + 21, f_all, f_limit, f_mean, @@ -1099,6 +1115,8 @@ cleri_grammar_t * compile_siri_grammar_grammar(void) f_stddev, f_first, f_last, + f_timeval, + f_interval, f_difference, f_derivative, f_filter, -- 2.30.2